
% comparison of hyps learners vs exact:
% check how well each does over a range of hyperparameters
% uses cvwl

addpath('gpml-matlab-v3.1-2010-09-27');
startup;
warning('off','all'); % gotta love playing with fire
n_expts = 5;
n_reps = 3;
rn = 9;
n = rn*rn;

results = zeros(n_expts, 5); % frac right, error, for each method; and then mean square error of hyps
data = zeros(n_reps,2); % how well hetero does each time; how well homo does
for expt = 1:n_expts
  fprintf('\nExpt: %i\n', expt);
  for trial = 1:n_reps
    fprintf('Trial: %i\n',trial);
    gendata(normrnd([0; 0.8 - trial / 3.0; 0], 0.2), rn);
    
    disp('CVWL')
    cross_validate_with_learning();
    tmp = load('cv_results');
    data(trial, 1) = tmp(1);

    disp('CV')
    cross_validate();
    tmp = load('cv_results');
    data(trial, 2) = tmp(1);

  end
  av = sum(data(:,1)) / n_reps;
  results(expt,1) = av;
  vars = (data(:,1) - av) .^ 2;
  results(expt,2) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
  av = sum(data(:,2)) / n_reps;
  results(expt,3) = av;
  vars = (data(:,2) - av) .^ 2;
  results(expt,4) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
  tmp = load('learnt_hyps_error');
  results(expt, 5) = tmp;
  
  dlmwrite('expt5_results', results);
end
results = results/n; % normalise
results(:,5) *= n; % not that part
dlmwrite('expt5_results', results);
% note that it's in row form: WL fraction correct, the variance thereof, W/O learning fraction correct, the variance thereof
disp('Done');